# Current method: the main difference between the OLD method and the current method is that
# they use different bounds. :)
# Note: also, the reason why this two tables can NOT generate at the same time is that the table:molecular need to use the table:denominator's info
# Basically, what term which cell belonging to.

# current method
# step-2 load the aux file for the small buckets info
tupleRangeWithIDDict = {}
inputAuxFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/smallBucketsRangesForQueryTermOverGov2Dataset_withUniqueIDAdded"
inputAuxFileHandler = open(inputAuxFileName,"r")
for line in inputAuxFileHandler.readlines():
    lineElements = line.strip().split(" ")
    uniqueID = lineElements[0]
    currentLowerBound = int(lineElements[3])
    currentUpperBound = int(lineElements[4])
    keyTuple = (currentLowerBound,currentUpperBound)
    if keyTuple not in tupleRangeWithIDDict:
        tupleRangeWithIDDict[keyTuple] = uniqueID
    else:
        print "Unexpected Behaviour"
        exit(1)

print "len(tupleRangeWithIDDict):",len(tupleRangeWithIDDict)
inputAuxFileHandler.close()

# step-1
# need to load the real freq in queries in order to filter out the query term which has freq >= 20
# so there might be some new query terms which does not exist in this dict and need to handle this
trainingQueryTermsWithTheirFreqInQueries = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsInQueries_head_85K_0_85%_sortedByQueryTermFreq"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    realFreqInQueries = int(lineElements[1])
    if queryTerm not in trainingQueryTermsWithTheirFreqInQueries:
        trainingQueryTermsWithTheirFreqInQueries[queryTerm] = realFreqInQueries

print "len(trainingQueryTermsWithTheirFreqInQueries):",len(trainingQueryTermsWithTheirFreqInQueries)
inputFileHandler.close()


# step0:
# load the aux file for the query terms mainly containing the length of the inverted list  
allQueryTermsWithTheirFreqInCollection = {}
inputFileAuxFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueryTermsWithTermFreqInCollection.txt"
inputFileAuxFileHandler = open(inputFileAuxFileName,"r")

for line in inputFileAuxFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    freqInCollectionForTheQueryTerm = int( lineElements[1] )
    if queryTerm not in allQueryTermsWithTheirFreqInCollection:
        allQueryTermsWithTheirFreqInCollection[queryTerm] = freqInCollectionForTheQueryTerm
    else:
        print "Unexpected Behaviour"
        exit(1)

print "len(allQueryTermsWithTheirFreqInCollection):",len(allQueryTermsWithTheirFreqInCollection)
inputFileAuxFileHandler.close()


# step1:
queryTermSpeciesDict = {}
speciesNameWithCounterDictForJustificationSet = {}
classWithTheirQueryTermListDict = {}

classLabelList = []
for i in range(0,1000):
    classLabelList.append( str(i) )
print "len(classLabelList):",len(classLabelList)
for i in range(0,20):
    for classLabel in classLabelList:
        key = str(i) + "_" + classLabel
        speciesNameWithCounterDictForJustificationSet[key] = 0
        classWithTheirQueryTermListDict[key] = []

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/freqOfFreqInQueries_head_85K_0_85%_2D_with_query_terms.txt"
inputFileHandler = open(inputFileName,"r")
for index,line in enumerate( inputFileHandler.readlines() ):
    # print index
    lineElements = line.strip().split(" ")
    speciesName = lineElements[0]
    numOfKindsInThisSpecies = int( lineElements[1] )
    
    for queryTerm in lineElements[2:]:
        if queryTerm not in queryTermSpeciesDict:
            # speciesName is also the key
            queryTermSpeciesDict[queryTerm] = speciesName
        else:
            print "mark2, unexpected"
            exit(1)

print "len(queryTermSpeciesDict):",len(queryTermSpeciesDict)
# for debug check only
# print "queryTermSpeciesDict:",queryTermSpeciesDict
inputFileHandler.close()


# step2:
dictForQueryTermHaveSeen = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/100KQueries_1_10%"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/100KQueries_1_10%_without_query_terms"
outputFileHandler = open(outputFileName,"w")

for line in inputFileHandler.readlines():
        # print "line:",line.strip()
        queryTermList = line.strip().split(":")[1].strip().split(" ")
        # print "queryTermList:",queryTermList
        
        data = ""
        for element in queryTermList:
            data += element + " "
        
        # print "data(old):",data
        # print "original data:",data
        
        for i in range(0,len(data)):
            # print "data[i]:",ord(data[i])
            if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
                # Just replace them with a space.
                data = data[:i] + " " + data[i+1:]
    
        # print "data(new):",data
        
        currentNewQueryTermList = data.strip().split(" ")
        currentNewQueryTermDict = {}
        
        for queryTerm in currentNewQueryTermList:
            if queryTerm.strip() != "":
                queryTermLower = queryTerm.lower()
                if queryTermLower not in currentNewQueryTermDict:
                    currentNewQueryTermDict[queryTermLower] = 1

        # Let's do the new thing here for the alg. of prof
        for queryTerm in currentNewQueryTermDict:
            if queryTerm not in dictForQueryTermHaveSeen:
                
                dictForQueryTermHaveSeen[queryTerm] = 1
                
                if queryTerm in queryTermSpeciesDict:
                    # This queryTerm actually belongs to a specific species
                    speciesNameWithCounterDictForJustificationSet[ queryTermSpeciesDict[queryTerm] ] += 1
                    classWithTheirQueryTermListDict[ queryTermSpeciesDict[queryTerm] ].append(queryTerm)
                else:
                    # One of the conditions is that ONLY deal with the query terms which has freq less than 20. The others, just ignored.
                    if queryTerm not in trainingQueryTermsWithTheirFreqInQueries or trainingQueryTermsWithTheirFreqInQueries[queryTerm] < 20:
                        # debug purpose
                        # print queryTerm,"strange"
                        
                        # the freq label is 0 and what about the label from the document distribution part
                        currentTwoDClass = "N/A"
                        
                        lengthOfListForLexiconTerm = allQueryTermsWithTheirFreqInCollection[queryTerm]
                        if lengthOfListForLexiconTerm < 1:
                            pass
                        else:
                            # current version with small buckets.
                            for tuple in tupleRangeWithIDDict:
                                (currentLowerBound,currentUpperBound) = tuple
                                if lengthOfListForLexiconTerm >= currentLowerBound and lengthOfListForLexiconTerm < currentUpperBound:
                                    currentTwoDClass = tupleRangeWithIDDict[tuple]
                                    break
                            
                            '''
                            # old version
                            if lengthOfListForLexiconTerm >= 1 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE1:
                                # it is very rare
                                # example:
                                # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                                currentTwoDClass = "VR"
                    
                            elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE1 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE2:
                                # it is not frequent
                                # example:
                                # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                                currentTwoDClass = "NF"
                    
                            elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE2 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE3:
                                # it is medium
                                # example:
                                # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                                currentTwoDClass = "M"
                    
                            elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE3 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE4:
                                # it is frequent
                                # example:
                                # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                                currentTwoDClass = "F"
                    
                            elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE4:
                                # it is very frequent
                                # example:
                                # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                                currentTwoDClass = "VF"
                            '''
                            
                            currentTwoDClass = "0" + "_" + currentTwoDClass 
                            
                            speciesNameWithCounterDictForJustificationSet[ currentTwoDClass ] += 1 
                            classWithTheirQueryTermListDict[currentTwoDClass].append(queryTerm)
                    else:
                        # freq >= 20 which we don't care currently.
                        pass
            else:
                # because this term has been seen in the justification set
                pass

print "len(classWithTheirQueryTermListDict):",len(classWithTheirQueryTermListDict)
print "len(speciesNameWithCounterDictForJustificationSet):",len(speciesNameWithCounterDictForJustificationSet)

for key in classWithTheirQueryTermListDict:
    print key,len( classWithTheirQueryTermListDict[key] ),speciesNameWithCounterDictForJustificationSet[key]
    if len( classWithTheirQueryTermListDict[key] ) != speciesNameWithCounterDictForJustificationSet[key]:
        print "NOT Expected Behavior"
        exit(1)


for key in speciesNameWithCounterDictForJustificationSet:
    outputLine = key + " " + str(speciesNameWithCounterDictForJustificationSet[key]) + " "
    '''
    if key in classWithTheirQueryTermListDict:
        for queryTerm in classWithTheirQueryTermListDict[key]:
            outputLine += queryTerm + " "
    '''
    outputLine += "\n"
    outputFileHandler.write(outputLine)

outputFileHandler.close()
inputFileHandler.close()

'''
# OLD method
UPPER_BOUND_FOR_RANGE1 = 100
UPPER_BOUND_FOR_RANGE2 = 5000
UPPER_BOUND_FOR_RANGE3 = 80000
UPPER_BOUND_FOR_RANGE4 = 600000

# step-1
# need to load the real freq in queries in order to filter out the query term which has freq >= 20
# so there might be some new query terms which does not exist in this dict and need to handle this
trainingQueryTermsWithTheirFreqInQueries = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/realFreqOfTermsInQueries_head_85K_0_85%_sortedByQueryTermFreq"
inputFileHandler = open(inputFileName,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    realFreqInQueries = int(lineElements[1])
    if queryTerm not in trainingQueryTermsWithTheirFreqInQueries:
        trainingQueryTermsWithTheirFreqInQueries[queryTerm] = realFreqInQueries

print "len(trainingQueryTermsWithTheirFreqInQueries):",len(trainingQueryTermsWithTheirFreqInQueries)
inputFileHandler.close()


# step0:
# load the aux file for the query terms mainly containing the length of the inverted list  
allQueryTermsWithTheirFreqInCollection = {}
inputFileAuxFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/100KQueryTermsWithTermFreqInCollection.txt"
inputFileAuxFileHandler = open(inputFileAuxFileName,"r")

for line in inputFileAuxFileHandler.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    freqInCollectionForTheQueryTerm = int( lineElements[1] )
    if queryTerm not in allQueryTermsWithTheirFreqInCollection:
        allQueryTermsWithTheirFreqInCollection[queryTerm] = freqInCollectionForTheQueryTerm
    else:
        print "Unexpected Behaviour"
        exit(1)

print "len(allQueryTermsWithTheirFreqInCollection):",len(allQueryTermsWithTheirFreqInCollection)
inputFileAuxFileHandler.close()


# step1:
queryTermSpeciesDict = {}
speciesNameWithCounterDictForJustificationSet = {}
classWithTheirQueryTermListDict = {}
classLabelList = ["VR","NF","M","F","VF"]
for i in range(0,20):
    for classLabel in classLabelList:
        key = str(i) + "_" + classLabel
        speciesNameWithCounterDictForJustificationSet[key] = 0
        classWithTheirQueryTermListDict[key] = []

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/freqOfFreqInQueries_head_85K_0_85%_2D_with_query_terms.txt"
inputFileHandler = open(inputFileName,"r")
for index,line in enumerate( inputFileHandler.readlines() ):
    # print index
    lineElements = line.strip().split(" ")
    speciesName = lineElements[0]
    numOfKindsInThisSpecies = int( lineElements[1] )
    
    for queryTerm in lineElements[2:]:
        if queryTerm not in queryTermSpeciesDict:
            # speciesName is also the key
            queryTermSpeciesDict[queryTerm] = speciesName
        else:
            print "mark2, unexpected"
            exit(1)

print "len(queryTermSpeciesDict):",len(queryTermSpeciesDict)
# for debug check only
# print "queryTermSpeciesDict:",queryTermSpeciesDict
inputFileHandler.close()


# step2:
dictForQueryTermHaveSeen = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/100KQueries_1_10%"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/100KQueries_1_10%_with_query_terms"
outputFileHandler = open(outputFileName,"w")

for line in inputFileHandler.readlines():
        # print "line:",line.strip()
        queryTermList = line.strip().split(":")[1].strip().split(" ")
        # print "queryTermList:",queryTermList
        
        data = ""
        for element in queryTermList:
            data += element + " "
        
        # print "data(old):",data
        # print "original data:",data
        
        for i in range(0,len(data)):
            # print "data[i]:",ord(data[i])
            if not ( (ord(data[i]) >= 48 and ord(data[i]) < 58) or (ord(data[i]) >= 65 and ord(data[i]) < 91) or (ord(data[i]) >= 97 and ord(data[i]) < 123) or (ord(data[i]) == 32) ):
                # Just replace them with a space.
                data = data[:i] + " " + data[i+1:]
    
        # print "data(new):",data
        
        currentNewQueryTermList = data.strip().split(" ")
        currentNewQueryTermDict = {}
        
        for queryTerm in currentNewQueryTermList:
            if queryTerm.strip() != "":
                queryTermLower = queryTerm.lower()
                if queryTermLower not in currentNewQueryTermDict:
                    currentNewQueryTermDict[queryTermLower] = 1

        # Let's do the new thing here for the alg. of prof
        for queryTerm in currentNewQueryTermDict:
            if queryTerm not in dictForQueryTermHaveSeen:
                
                dictForQueryTermHaveSeen[queryTerm] = 1
                
                if queryTerm in queryTermSpeciesDict:
                    # This queryTerm actually belongs to a specific species
                    speciesNameWithCounterDictForJustificationSet[ queryTermSpeciesDict[queryTerm] ] += 1
                    classWithTheirQueryTermListDict[ queryTermSpeciesDict[queryTerm] ].append(queryTerm)
                else:
                    if queryTerm not in trainingQueryTermsWithTheirFreqInQueries or trainingQueryTermsWithTheirFreqInQueries[queryTerm] < 20:
                        # debug purpose
                        # print queryTerm,"strange"
                        
                        # the freq label is 0 and what about the label from the document distribution part
                        currentTwoDClass = "N/A"
                        
                        lengthOfListForLexiconTerm = allQueryTermsWithTheirFreqInCollection[queryTerm]
                        if lengthOfListForLexiconTerm < 1:
                            pass
                        else:
                            if lengthOfListForLexiconTerm >= 1 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE1:
                                # it is very rare
                                # example:
                                # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                                currentTwoDClass = "VR"
                    
                            elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE1 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE2:
                                # it is not frequent
                                # example:
                                # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                                currentTwoDClass = "NF"
                    
                            elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE2 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE3:
                                # it is medium
                                # example:
                                # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                                currentTwoDClass = "M"
                    
                            elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE3 and lengthOfListForLexiconTerm < UPPER_BOUND_FOR_RANGE4:
                                # it is frequent
                                # example:
                                # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                                currentTwoDClass = "F"
                    
                            elif lengthOfListForLexiconTerm >= UPPER_BOUND_FOR_RANGE4:
                                # it is very frequent
                                # example:
                                # (freqOfFreqNr,modifiedFreqRStar,VFProbability,FProbability,MProbability,NFProbability,VRProbability)
                                currentTwoDClass = "VF"
                            
                            currentTwoDClass = "0" + "_" + currentTwoDClass 
                            
                            speciesNameWithCounterDictForJustificationSet[ currentTwoDClass ] += 1 
                            classWithTheirQueryTermListDict[currentTwoDClass].append(queryTerm)
                    else:
                        # freq >= 20 which we don't care currently.
                        pass
            else:
                # because this term has been seen in the justification set
                pass

print "len(classWithTheirQueryTermListDict):",len(classWithTheirQueryTermListDict)
print "len(speciesNameWithCounterDictForJustificationSet):",len(speciesNameWithCounterDictForJustificationSet)

for key in classWithTheirQueryTermListDict:
    print key,len( classWithTheirQueryTermListDict[key] ),speciesNameWithCounterDictForJustificationSet[key]
    if len( classWithTheirQueryTermListDict[key] ) != speciesNameWithCounterDictForJustificationSet[key]:
        print "NOT Expected Behavior"
        exit(1)


for key in speciesNameWithCounterDictForJustificationSet:
    outputLine = key + " " + str(speciesNameWithCounterDictForJustificationSet[key]) + " "
    
    if key in classWithTheirQueryTermListDict:
        for queryTerm in classWithTheirQueryTermListDict[key]:
            outputLine += queryTerm + " "
    
    outputLine += "\n"
    outputFileHandler.write(outputLine)

outputFileHandler.close()
inputFileHandler.close()
'''
